import requests
from lxml import etree
import random
from header_random import header_list
import re
headers={
    "User-Agent": random.choice(header_list),
    "Referer": "https://www.biqiuge.com/paihangbang/"
}

fiction_id = 0  # 小说名字id
zhangjie_id = 0

class Biqu():
    nav_id=0#导航id

    def Nav(self,urls):#爬取导航
        response = requests.get(url=urls,headers=headers)
        html = etree.HTML(response.text)
        all_navs = html.xpath("//div[@class='nav']/ul/li/a")[2:-2]
        for i in all_navs:
            # time.sleep(random.randint(2,3))
            self.nav_id=self.nav_id+1
            navs = i.xpath('./text()')[0]

            navs_href = urls+ i.xpath('./@href')[0]

            yield navs_href,navs,self.nav_id


            # self.Fiction(navs_href,self.nav_id)




    def Fiction(self,navs_href,nav_id):#爬取导航中小说名称
        global fiction_id
        response = requests.get(url=navs_href,headers=headers)
        html = etree.HTML(response.text)
        all_fiction = html.xpath("//div[@class='l']/ul/li/span[@class='s2']/a")[0:20]

        for i in all_fiction:
            # time.sleep(random.randint(2,3))
            fiction_id=fiction_id+1
            fiction_title = i.xpath("./text()")[0]
            fiction_href = i.xpath('./@href')[0]

            yield fiction_id,fiction_title,fiction_href,nav_id
            # self.gic_names(fiction_href,fiction_id,nav_id)


    def gic_names(self,fiction_href,fiction_id,nav_id):#爬取每个小说的信息
        response = requests.get(url=fiction_href,headers=headers)
        response.encoding = response.apparent_encoding
        html = etree.HTML(response.text)
        jianjie = html.xpath("string(//div[@id='maininfo']/div[@id='intro']/p[2])")
        author = html.xpath("//div[@id='maininfo']/div[@id='info']/p[1]/text()")[0]
        images = html.xpath("//div[@id='fmimg']/img/@src")[0]
        create_time = html.xpath("//div[@id='maininfo']/div[@id='info']/p[3]/text()")[0]
        fiction_title = html.xpath("//div[@id='maininfo']/div[@id='info']/h1/text()")[0].strip()

        yield fiction_id,fiction_title,fiction_href,jianjie,author,images,create_time,nav_id

    def zhangjie_name(self, hrefs, fiction_id):
        global zhangjie_id
        response = requests.get(url=hrefs,headers=headers)
        response.encoding = response.apparent_encoding
        html = etree.HTML(response.text)
        zhangjie_title = html.xpath("//div[@id='list']/dl/dd/a")[0:20]
        try:
            for i in zhangjie_title:
                title = i.xpath('./text()')[0]
                hrefs = "http://www.xbiquge.la"+i.xpath('./@href')[0]
                zhangjie_id=zhangjie_id+1
                yield  zhangjie_id,title,hrefs,fiction_id
                # self.text_name(zhangjie_id,hrefs)
        except Exception as e:
            print(e)
    def text_name(self,zhangjie_id,hrefs):

        response = requests.get(url=hrefs, headers=headers)
        response.encoding = response.apparent_encoding
        # time.sleep(random.randint(2, 3))

        html = etree.HTML(response.text)
        fonts =html.xpath('string(//*[@id="content"])')
        titles = html.xpath("string(//div[@class='box_con']/div[@class='bookname']/h1)")
        yield  zhangjie_id,titles, fonts, hrefs















if __name__=="__main__":
    pass
    # urls = 'http://www.xbiquge.la'
    # res = Biqu()
    # res.Nav(urls=urls)

